# -*- coding:utf-8 -*-
# @author: LY
# @time:2022/4/23 0023:9:23
import datetime
import pdb
import re, warnings, time, os
import threading

os.chdir(os.path.dirname(__file__))

import requests, json
warnings.filterwarnings("ignore")

from queue import Queue
from threading import Thread

from config_set.conn_pool import *
from config_set.operation_mysql import *
class beian_obj():
    def __init__(self):
        beian = {
            'host': '101.35.198.253',
            'port': 3306,
            'user': 'beian',
            'password': 'LeFsP8exKyYLJR2F',
            'database': 'beian',
            'charset': 'utf8'
        }
        while True:
            try:
                self.mysqlobj = get_connection(beian)  # mongodb 库连接对象
                break
            except:
                print("数据库重连")
                time.sleep(3)
        self.interface_que = Queue(maxsize=100)
        self.mysql_que = Queue(maxsize=100)
        self.thread_num = 20
        self.lock = threading.Lock()
        self.oper_data_num = 0
        self.stop = []
        self.version = "20220423"
        self.live_thread = []

    #入接口
    def save_interface(self):
        while True:
            try:
                temporary = self.interface_que.get(timeout=5*60)
            except:
                print("超时结束")
                return
            if not temporary and self.interface_que.empty():
                time.sleep(20)
                if self.interface_que.empty():
                    print("入库结束")
                    return
                else:
                    pass
            url_inface = "http://midd.com/api/comp_address/add_comp_address"
            # url_inface = "https://midd.jianshequan.com/api/comp_address/add_comp_address"
            header = {"Authorization": "Bearer kasjdflqjkeofawejifjfglewjemrgopwekrgopwerjkgopqwerjksgogkp"}
            while True:
                try:
                    response = requests.post(url=url_inface, headers=header, data=temporary, verify=False, allow_redirects=False)
                    if response.status_code >= 500:
                        time.sleep(1)
                        continue
                    break
                except Exception as e:
                    print(f"请求错误：{e}")
                    time.sleep(1)
            try:
                # print(response.json(), datetime.datetime.now())
                if "version" in response.json().get("msg") or response.json().get("msg") != "success":
                    print(response.json())
            except:
                print(response.text)

    def select_data(self, sorc_plat, thread_num):
        print(sorc_plat,"开始")
        mysqlobj = self.mysqlobj.connection()
        curr = mysqlobj.cursor()
        while True:
            select_sql = f"select id, pagenum from beian_data_new where source_name = '{sorc_plat}' ORDER BY id limit 1"
            select_result = conn_select(select_sql, curr)
            if select_result == "重连":
                try:
                    mysqlobj.close()
                    curr.close()
                except:
                    pass
                mysqlobj = self.mysqlobj.connection()
                curr = mysqlobj.cursor()
                continue
            elif select_result:
                save_num = int(select_result[0][1])
                start_id = int(select_result[0][0])
            else:
                save_num = 0
                start_id = 0
            count_sql = f"select count(1) from beian_data_new where source_name = '{sorc_plat}'"
            select_result = conn_select(count_sql, curr)
            if select_result == "重连":
                try:
                    mysqlobj.close()
                    curr.close()
                except:
                    pass
                mysqlobj = self.mysqlobj.connection()
                curr = mysqlobj.cursor()
                continue
            else:
                count_num = int(select_result[0][0])
            break
        ids = 0
        continue_num = 0
        select_num = 200
        oper_num = 0
        while True:
            numbers = 0
            temporary = {}
            select_data_sql = f"select * from beian_data_new where id > {ids} and source_name = '{sorc_plat}' limit {select_num}" #limit 1
            select_data_res = conn_select(select_data_sql, curr)
            # print(f"{sorc_plat} {len(select_data_res)} 数据量：{count_num}  操作数：{oper_num}  进度：{round(oper_num/count_num, 2)}%")
            if select_data_res:
                continue_num += len(select_data_res)
                oper_num += len(select_data_res)
                if continue_num < save_num:
                    ids = select_data_res[-1][0]
                    continue
                for every_data in select_data_res:
                    data_tuple = every_data
                    if numbers == 0:
                        temporary[f"version"] = self.version
                        temporary[f"beian_province_name"] = data_tuple[3].replace("新疆维吾尔族自治区", "新疆维吾尔自治区")
                        temporary[f"source_site"] = data_tuple[5]
                    data_dict = json.loads(data_tuple[1])
                    # if data_tuple[11]) == 1 or not data_tuple[12]:
                    #     continue
                    if not data_dict.get("comp_name"):
                        continue
                    if len(data_dict.get("comp_name")) <= 3:
                        continue
                    temporary[f"beian_comp[{numbers}][beian_type]"] = 1 if data_tuple[4] == "本省" else 2
                    comp_name = re.sub("\s", "", data_dict["comp_name"])
                    credit_no = ""
                    if data_dict.get("credit_no"):
                        if data_dict.get("credit_no").strip():
                            credit_no = data_dict["credit_no"].strip().upper()
                            credit_no = re.sub("[\(（].*?[）\)]|[^a-zA-Z0-9]|\s", "", credit_no)
                            if credit_no != "" and not re.search("[\u4e00-\u9fa5]", credit_no):
                                credit_no = credit_no.upper()
                    if not comp_name and not credit_no:
                        print("【", data_tuple, "】")
                        continue
                    temporary[f"beian_comp[{numbers}][credit_no]"] = credit_no
                    temporary[f"beian_comp[{numbers}][comp_name]"] = comp_name.replace("(", "（").replace(")", "）")
                    # temporary[f"beian_comp[{numbers}][oper_name]"] = data_dict["oper_name"].strip() if data_dict.get("oper_name") else ""
                    if data_tuple[11]:
                        temporary[f"beian_comp[{numbers}][comp_keyno]"] = data_tuple[11] if data_tuple[11] else ""
                    temporary[f"beian_comp[{numbers}][beian_unid]"] = data_tuple[8]
                    # if data_dict["create_date"]:
                    #     temporary[f"beian_comp[{numbers}][create_date]"] = data_dict["create_date"]
                    # else:
                    #     temporary[f"beian_comp[{numbers}][create_date]"] = data_dict["update_date"]
                    # temporary[f"beian_comp[{numbers}][update_date]"] = data_dict["update_date"]
                    # temporary[f"beian_comp[{numbers}][new_data]"] = data_dict["new_data"]
                    numbers += 1
                    save_num += 1
                ids = select_data_res[-1][0]
                self.interface_que.put(temporary)
                insert_sql = f"update beian_data_new set pagenum = {save_num} where id = {start_id}"
                conn_update(insert_sql, curr, mysqlobj)
            else:
                if temporary != {}:
                    self.interface_que.put(temporary)
                insert_sql = f"update beian_data_new set pagenum = 0 where id = {start_id}"
                conn_update(insert_sql, curr, mysqlobj)
                break
        with self.lock:
            self.oper_data_num += save_num
        # print()
        print(sorc_plat, "结束", "操作数据： ", oper_num, "条", "入接口数据： ", save_num, "条", "共入接口数据：", self.oper_data_num, "条")
        # print(len(threading.enumerate()))
        # for every_thread in threading.enumerate():
        #     if every_thread.getName() == sorc_plat:
        #         continue
        #     print(every_thread.getName(), end=" 存活  ；")
        print()

    # def check_thread(self, sorc_plat):
    #     self.live_thread

    def run(self, sorc_list):
        print(len(sorc_list))
        thread_list = []
        time.sleep(1)
        save_interface = Thread(target=beian.save_interface)
        # save_interface2 = Thread(target=beian.save_interface)
        for sorc_plar in sorc_list:
            beian_thread = Thread(target=beian.select_data, args=(sorc_plar, len(sorc_list),), name=sorc_plar)
            thread_list.append(beian_thread)
            # break

        thread_list.append(save_interface)
        # thread_list.append(save_interface2)
        for threads in thread_list:
            threads.start()
            time.sleep(0.2)

        for threadj in thread_list:
            threadj.join()
        print("结束时间：", datetime.datetime.now())


if __name__ == '__main__':
    beian = beian_obj()
    sorc_list = ['安徽省住房和城乡建设厅', '北京市住房和城乡建设委员会', '山西省智慧建筑管理服务信息平台', '河北省住房和城乡建设厅', '内蒙古自治区建筑市场监管与诚信信息平台', '辽宁省建筑市场公共服务平台', '吉林省建筑市场监管公共服务平台', '福建省建设行业信息公开平台', '江西住建云', '河南省建筑市场监管公共服务平台', '广东省建设行业数据开放平台', '海南省建筑市场信息公开平台', '广西建筑市场监管云平台', '重庆建设工程信息网', '云南省建筑市场监管与诚信信息网', '西藏自治区建筑市场监管与诚信信息平台', '外省入陕建筑业企业登记信息查询', '甘肃省住房和城乡建设厅', '青海省工程建设监管和信用管理平台', '宁夏建筑市场监管服务系统', '新疆工程建设云', '上海市建设市场信息服务平台', '山东省住房城乡建设服务监管与信用信息综合平台', '湖北省建筑市场监督与诚信一体化工作平台', '湖北省建筑市场监督与诚信一体化平台', '浙江省建筑市场监管公共服务系统']
    beian.run(sorc_list)

